I am trying to actually study the signal handling behavior in multiprocess system. I have a system where there are three signal generating processes generating signals of type SIGUSR1 and SIGUSR1. I have two handler processes that handle a particular type of signal. I have another monitoring process that also receives the signals and then does its work. I have a certain issue. Whenever my signal handling processes generate a signal of a particular type, it is sent to the process group so it is received by the signal handling processes as well as the monitoring processes.

Whenever the signal handlers of monitoring and signal handling processes are called, I have printed to indicate the signal handling. I was expecting a uniform series of calls for the signal handlers of the monitoring and handling processes. However, looking at the output I could see like at the beginning the monitoring and signal handling processes's signal handlers are called uniformly.

However, after I could see like signal handler processes handlers being called in a burst followed by the signal handler of monitoring process being called in a burst.

Here is my code and output

#include <iostream>
#include <sys/types.h>
#include <sys/wait.h>
#include <sys/time.h>
#include <signal.h>
#include <cstdio>
#include <stdlib.h>
#include <sys/ipc.h>
#include <sys/shm.h>

#define NUM_SENDER_PROCESSES 3
#define NUM_HANDLER_PROCESSES 4
#define NUM_SIGNAL_REPORT 10
#define MAX_SIGNAL_COUNT 100000

using namespace std;

volatile int *usrsig1_handler_count;
volatile int *usrsig2_handler_count;
volatile int *usrsig1_sender_count;
volatile int *usrsig2_sender_count;
volatile int *lock_1;
volatile int *lock_2;
volatile int *lock_3;
volatile int *lock_4;
volatile int *lock_5;
volatile int *lock_6;


//Used only by the monitoring process
volatile int monitor_count;
volatile int usrsig1_monitor_count;
volatile int usrsig2_monitor_count;
double time_1[NUM_SIGNAL_REPORT];
double time_2[NUM_SIGNAL_REPORT];

//Used only by the main process
int total_signal_count;

//For shared memory
int shmid;
const int shareSize = sizeof(int) * (10);



double timestamp() {
  struct timeval tp;
  gettimeofday(&tp, NULL);
  return (double)tp.tv_sec + tp.tv_usec / 1000000.;
}


pid_t senders[NUM_SENDER_PROCESSES];
pid_t handlers[NUM_HANDLER_PROCESSES];
pid_t reporter;

void signal_catcher_1(int);
void signal_catcher_2(int);
void signal_catcher_int(int);
void signal_catcher_monitor(int);
void signal_catcher_main(int);

void terminate_processes() {
  //Kill the child processes
  int status;
  cout << "Time up terminating the child processes" << endl;
  for(int i=0; i<NUM_SENDER_PROCESSES; i++) {
    kill(senders[i],SIGKILL);
  }
  for(int i=0; i<NUM_HANDLER_PROCESSES; i++) {
    kill(handlers[i],SIGKILL);
  }
  kill(reporter,SIGKILL);

  //Wait for the child processes to finish
  for(int i=0; i<NUM_SENDER_PROCESSES; i++) {
    waitpid(senders[i], &status, 0);
  }
  for(int i=0; i<NUM_HANDLER_PROCESSES; i++) {
    waitpid(handlers[i], &status, 0);
  }
  waitpid(reporter, &status, 0);
}

int main(int argc, char *argv[]) {
  if(argc != 2) {
    cout << "Required parameters missing. " << endl;
    cout << "Option 1 = 1 which means run for 30 seconds" << endl;
    cout << "Option 2 = 2 which means run until 100000 signals" << endl;
    exit(0);
  }

  int option = atoi(argv[1]);

  pid_t pid;

  if(option == 2) {
    if(signal(SIGUSR1, signal_catcher_main) == SIG_ERR) {
      perror("1");
      exit(1);
    }

    if(signal(SIGUSR2, signal_catcher_main) == SIG_ERR) {
      perror("2");
      exit(1);
    }
  } else {
    if(signal(SIGUSR1, SIG_IGN) == SIG_ERR) {
      perror("1");
      exit(1);
    }

    if(signal(SIGUSR2, SIG_IGN) == SIG_ERR) {
      perror("2");
      exit(1);
    }

  }

  if(signal(SIGINT, signal_catcher_int) == SIG_ERR) {
    perror("3");
    exit(1);
  }

  ///////////////////////////////////////////////////////////////////////////////////////
  ////////////////////// Initializing the shared memory /////////////////////////////////
  ///////////////////////////////////////////////////////////////////////////////////////
  cout << "Initializing the shared memory" << endl;
  if ((shmid=shmget(IPC_PRIVATE,shareSize,IPC_CREAT|0660))< 0) {
    perror("shmget fail");
    exit(1);
  }  

  usrsig1_handler_count = (int *) shmat(shmid, NULL, 0);
  usrsig2_handler_count = usrsig1_handler_count + 1;
  usrsig1_sender_count = usrsig2_handler_count + 1;
  usrsig2_sender_count = usrsig1_sender_count + 1;
  lock_1 = usrsig2_sender_count + 1;
  lock_2 = lock_1 + 1;
  lock_3 = lock_2 + 1;
  lock_4 = lock_3 + 1;
  lock_5 = lock_4 + 1;
  lock_6 = lock_5 + 1;

  //Initialize them to be zero
  *usrsig1_handler_count = 0;
  *usrsig2_handler_count = 0;
  *usrsig1_sender_count = 0;
  *usrsig2_sender_count = 0;
  *lock_1 = 0;
  *lock_2 = 0;
  *lock_3 = 0;
  *lock_4 = 0;
  *lock_5 = 0;
  *lock_6 = 0;


  cout << "End of initializing the shared memory" << endl;
  /////////////////////////////////////////////////////////////////////////////////////////////
  /////////////////// End of initializing the shared memory ///////////////////////////////////
  /////////////////////////////////////////////////////////////////////////////////////////////


  ///////////////////////////////////////////////////////////////////////////////////////////
  /////////////////////////////Registering the signal handlers///////////////////////////////
  ///////////////////////////////////////////////////////////////////////////////////////////
  cout << "Registering the signal handlers" << endl;
  for(int i=0; i<NUM_HANDLER_PROCESSES; i++) {
    if((pid = fork()) == 0) {
      if(i%2 == 0) {
        struct sigaction action;
        action.sa_handler = signal_catcher_1;
        sigset_t block_mask;
        action.sa_flags = 0;
        sigaction(SIGUSR1,&action,NULL);

        if(signal(SIGUSR2, SIG_IGN) == SIG_ERR) {
          perror("2");
          exit(1);
        }

      } else {
        if(signal(SIGUSR1 ,SIG_IGN) == SIG_ERR) {
          perror("1");
          exit(1);
        }

        struct sigaction action;
        action.sa_handler = signal_catcher_2;
        action.sa_flags = 0;
        sigaction(SIGUSR2,&action,NULL);

      }

      if(signal(SIGINT, SIG_DFL) == SIG_ERR) {
        perror("2");
        exit(1);
      }

      while(true) {
        pause();
      }
      exit(0);

    } else {
      //cout << "Registerd the handler " << pid << endl;
      handlers[i] = pid;
    }
  }
  cout << "End of registering the signal handlers" << endl;

  /////////////////////////////////////////////////////////////////////////////////////////////////////
  ////////////////////////////End of registering the signal handlers //////////////////////////////////
  /////////////////////////////////////////////////////////////////////////////////////////////////////


  ////////////////////////////////////////////////////////////////////////////////////////////////////
  ///////////////////////////Registering the monitoring process //////////////////////////////////////
  ////////////////////////////////////////////////////////////////////////////////////////////////////
  cout << "Registering the monitoring process" << endl;
  if((pid = fork()) == 0) {
    struct sigaction action;
    action.sa_handler = signal_catcher_monitor;
    sigemptyset(&action.sa_mask);
    sigset_t block_mask;
    sigemptyset(&block_mask);
    sigaddset(&block_mask,SIGUSR1);
    sigaddset(&block_mask,SIGUSR2);
    action.sa_flags = 0;
    action.sa_mask = block_mask;
    sigaction(SIGUSR1,&action,NULL);
    sigaction(SIGUSR2,&action,NULL);


    if(signal(SIGINT, SIG_DFL) == SIG_ERR) {
      perror("2");
      exit(1);
    }

    while(true) {
      pause();
    }
    exit(0);

  } else {
    cout << "Monitor's pid is " << pid << endl;
    reporter = pid;
  }
  cout << "End of registering the monitoring process" << endl;

  /////////////////////////////////////////////////////////////////////////////////////////////////////
  ////////////////////////End of registering the monitoring process////////////////////////////////////
  /////////////////////////////////////////////////////////////////////////////////////////////////////


  //Sleep to make sure that the monitor and handler processes are well initialized and ready to handle signals
  sleep(5);


  //////////////////////////////////////////////////////////////////////////////////////////////////////
  //////////////////////////Registering the signal generators///////////////////////////////////////////
  /////////////////////////////////////////////////////////////////////////////////////////////////////
  cout << "Registering the signal generators" << endl;
  for(int i=0; i<NUM_SENDER_PROCESSES; i++) {

    if((pid = fork()) == 0) {

      if(signal(SIGUSR1, SIG_IGN) == SIG_ERR) {
        perror("1");
        exit(1);
      }

      if(signal(SIGUSR2, SIG_IGN) == SIG_ERR) {
        perror("2");
        exit(1);
      }

      if(signal(SIGINT, SIG_DFL) == SIG_ERR) {
        perror("2");
        exit(1);
      }

      srand(i);        
      while(true) {
        int signal_id = rand()%2 + 1;

        if(signal_id == 1) {
          killpg(getpgid(getpid()), SIGUSR1);
          while(__sync_lock_test_and_set(lock_4,1) != 0) {
          }
          (*usrsig1_sender_count)++;
          *lock_4 = 0;
        } else {
          killpg(getpgid(getpid()), SIGUSR2);
          while(__sync_lock_test_and_set(lock_5,1) != 0) {
          }
          (*usrsig2_sender_count)++;
          *lock_5=0;
        }       
        int r = rand()%10 + 1;
        double s = (double)r/100;
        sleep(s);
      }

      exit(0); 
    } else {
      //cout << "Registered the sender " << pid << endl;
      senders[i] = pid;
    } 
  }

  //cout << "End of registering the signal generators" << endl;
  /////////////////////////////////////////////////////////////////////////////////////////////////////
  //////////////////////////End of registering the signal generators///////////////////////////////////
  /////////////////////////////////////////////////////////////////////////////////////////////////////


  //Either sleep for 30 seconds and terminate the program or if the number of signals generated reaches 10000, terminate the program
  if(option = 1) {
    sleep(90);
    terminate_processes(); 
  } else {
    while(true) {
      if(total_signal_count >= MAX_SIGNAL_COUNT) {
        terminate_processes();
      } else {
        sleep(0.001);
      }
    }
  }
}




void signal_catcher_1(int the_sig) {
  while(__sync_lock_test_and_set(lock_1,1) != 0) {
  }
  (*usrsig1_handler_count) = (*usrsig1_handler_count) + 1;
  cout << "Signal Handler 1 " << *usrsig1_handler_count << endl;
  __sync_lock_release(lock_1);
}


void signal_catcher_2(int the_sig) {
   while(__sync_lock_test_and_set(lock_2,1) != 0) {
   }
   (*usrsig2_handler_count) = (*usrsig2_handler_count) + 1;
   __sync_lock_release(lock_2);

}

void signal_catcher_main(int the_sig) {
  while(__sync_lock_test_and_set(lock_6,1) != 0) {
  }
  total_signal_count++;
  *lock_6 = 0;
}


void signal_catcher_int(int the_sig) {
  for(int i=0; i<NUM_SENDER_PROCESSES; i++) {
    kill(senders[i],SIGKILL);
  }

  for(int i=0; i<NUM_HANDLER_PROCESSES; i++) {
    kill(handlers[i],SIGKILL);
  }

  kill(reporter,SIGKILL);

  exit(3);
}

void signal_catcher_monitor(int the_sig) {
  cout << "Monitoring process " << *usrsig1_handler_count << endl;
}

Here is the initial segment of output

Monitoring process 0
Monitoring process 0
Monitoring process 0
Monitoring process 0
Signal Handler 1 1
Monitoring process 2
Signal Handler 1 2
Signal Handler 1 3
Signal Handler 1 4
Monitoring process 4
Monitoring process Signal Handler 1 6
Signal Handler 1 7
Monitoring process 7
Monitoring process 8
Monitoring process 8
Signal Handler 1 9
Monitoring process 9
Monitoring process 9
Monitoring process 10
Signal Handler 1 11
Monitoring process 11
Monitoring process 12
Signal Handler 1 13
Signal Handler 1 14
Signal Handler 1 15
Signal Handler 1 16
Signal Handler 1 17
Signal Handler 1 18
Monitoring process 19
Signal Handler 1 20
Monitoring process 20
Signal Handler 1 21
Monitoring process 21
Monitoring process 21
Monitoring process 22
Monitoring process 22
Monitoring process 23
Signal Handler 1 24
Signal Handler 1 25
Monitoring process 25
Signal Handler 1 27
Signal Handler 1 28
Signal Handler 1 29

Here is the segment when the signal handler processes signal handlers are called in a burst

Signal Handler 1 456
Signal Handler 1 457
Signal Handler 1 458
Signal Handler 1 459
Signal Handler 1 460
Signal Handler 1 461
Signal Handler 1 462
Signal Handler 1 463
Signal Handler 1 464
Signal Handler 1 465
Signal Handler 1 466
Signal Handler 1 467
Signal Handler 1 468
Signal Handler 1 469
Signal Handler 1 470
Signal Handler 1 471
Signal Handler 1 472
Signal Handler 1 473
Signal Handler 1 474
Signal Handler 1 475
Signal Handler 1 476
Signal Handler 1 477
Signal Handler 1 478
Signal Handler 1 479
Signal Handler 1 480
Signal Handler 1 481
Signal Handler 1 482
Signal Handler 1 483
Signal Handler 1 484
Signal Handler 1 485
Signal Handler 1 486
Signal Handler 1 487
Signal Handler 1 488
Signal Handler 1 489
Signal Handler 1 490
Signal Handler 1 491
Signal Handler 1 492
Signal Handler 1 493
Signal Handler 1 494
Signal Handler 1 495
Signal Handler 1 496
Signal Handler 1 497
Signal Handler 1 498
Signal Handler 1 499
Signal Handler 1 500
Signal Handler 1 501
Signal Handler 1 502
Signal Handler 1 503
Signal Handler 1 504
Signal Handler 1 505
Signal Handler 1 506

Here is the segment when the monitoring processes signal handlers are called in a burst

Monitoring process 140
Monitoring process 140
Monitoring process 140
Monitoring process 140
Monitoring process 140
Monitoring process 140
Monitoring process 140
Monitoring process 140
Monitoring process 140
Monitoring process 140
Monitoring process 140
Monitoring process 140
Monitoring process 140
Monitoring process 140
Monitoring process 140
Monitoring process 140
Monitoring process 140
Monitoring process 140
Monitoring process 140
Monitoring process 140
Monitoring process 140
Monitoring process 140
Monitoring process 140
Monitoring process 140
Monitoring process 140
Monitoring process 140
Monitoring process 140
Monitoring process 140
Monitoring process 140
Monitoring process 140
Monitoring process 140
Monitoring process 140
Monitoring process 140
Monitoring process 140
Monitoring process 140
Monitoring process 140
Monitoring process 140
Monitoring process 140
Monitoring process 140
Monitoring process 140
Monitoring process 140
Monitoring process 140
Monitoring process 140
Monitoring process 140
Monitoring process 140
Monitoring process 140
Monitoring process 140
Monitoring process 140

Why isn't it uniform afterwards. Why are they called in a burst?